package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.JianYangForumTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.CommonUtils;
import com.spider.silence.utils.DateUtils;
import com.spider.silence.utils.FileUtils;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.net.URLDecoder;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @author 夏茂轩
 * @package com.spider.silence.downloader
 * @Date 2017/1/16 10:42
 * @description 简阳论坛下载器
 */
public class JianYangForumDownLoader extends AbstractCommonDownLoader implements Cloneable {
    private Logger logger = Logger.getLogger(JianYangForumDownLoader.class);

    private Map<String, File> files = null;

    public Map<String, File> getFiles() {

        return files;
    }

    private String mainPage = "http://www.jy0832.com/forum.php";

    public void generateTask() {
        Document document = null;
        boolean flag = true;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("JianYang.TimeOut"))) {
            try {
                Page pageContent = null;
                pageContent = getHtml("http://www.jy0832.com/forum.php");
                if (null == pageContent) {

                } else if (pageContent.getStatusCode() == 0) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                } else if (pageContent.getStatusCode() == 200) {
                    flag = false;
                    document = pageContent.getHtml().getDocument();
                    Elements elements = document.select(".fl_g dl dt a");
                    ArrayList<ArrayList<Integer>> taskSplit = CommonUtils.splitPages(1, elements.size(), 50);
                    ExecutorService pool = Executors.newFixedThreadPool(taskSplit.size());
                    try {
                        for (int i = 0; i < taskSplit.size(); i++) {
                            ModuleCrawler crawler = new ModuleCrawler(elements, taskSplit.get(i));
                            pool.execute(crawler);//提交任务
                        }
                    } catch (Exception e) {
                        e.printStackTrace();
                    } finally {
                        pool.shutdown();
                    }
                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                } else {

                }

            } catch (Exception e1) {
                logger.error(e1.getMessage(), e1);
            } finally {
                nowTime = System.currentTimeMillis();
            }
        }
    }

    /**
     * @author 夏茂轩
     * @date 2017/1/16 11:50
     * @description 爬取论坛子模块下面的URL
     */
    class ModuleCrawler extends AbstractCommonDownLoader implements Runnable {
        private Logger logger = Logger.getLogger(ModuleCrawler.class);
        private Elements elements;
        private ArrayList<Integer> taskIndex;

        public ModuleCrawler(Elements elements, ArrayList<Integer> taskIndex) {
            this.elements = elements;
            this.taskIndex = taskIndex;
        }

        @Override
        public void run() {
            for (int k = 0; k < taskIndex.size(); k++) {
                int index = taskIndex.get(k) - 1;
                //只爬取模块下第一页
               /*
               String url = elements.get(index).attr("href");
               try {
                    Document document = get(url);
                    Elements forumElements = document.select("tbody[id^='normalthread_'] a.xst");
                    for (int i = 0; i < forumElements.size(); i++) {
                        JianYangForumTask task = new JianYangForumTask(forumElements.get(i).attr("href"));
                        CommonTaskManager.add("jianYang", task);
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                    e.printStackTrace();
                }*/
                String moduleUrl = elements.get(index).absUrl("href");
                try {
                    if (null != moduleUrl) {
                        boolean flag = true;
                        long startTime = System.currentTimeMillis();
                        long nowTime = startTime;
                        while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("JianYang.TimeOut"))) {
                            try {
                                Document document = null;
                                Page pageContent = null;
                                String getDataUrl = moduleUrl;
                                pageContent = getHtml(getDataUrl,true);
                                if (null == pageContent) {

                                } else if (pageContent.getStatusCode() == 0) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                                } else if (pageContent.getStatusCode() == 200) {
                                    flag = false;
                                    document = pageContent.getHtml().getDocument();
                                    document.select(".aimg_tip").remove();
                                    Elements forumElements = document.select("tbody[id^='normalthread_'] a.xst");
                                    for (int j = 0; j < forumElements.size(); j++) {
                                        JianYangForumTask task = new JianYangForumTask(forumElements.get(j).absUrl("href"));
                                        CommonTaskManager.add("jianYang", task);
                                    }
                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                                } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                                } else {

                                }

                            } catch (Exception e1) {
                                //System.out.println("异常信息："+e1.getMessage());
                                logger.error(e1.getMessage(), e1);
                            } finally {
                                nowTime = System.currentTimeMillis();
                            }
                        }
                    }
                } catch (Exception e) {
                    logger.error(e.getMessage());
                    continue;
                }


            }
        }
    }

    /**
     * @param url 帖子的链接
     * @return
     * @author 夏茂轩
     * @date 2017/1/16 13:46
     * @description 爬取简阳论坛具体的帖子
     */
    public JSONObject crawlItem(String url) {
        files = new ConcurrentHashMap<>();
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();

        int page = 1;
        //System.out.println("爬取的：" + url);
        try {
            // Document document=get(url,true);
            Document document = null;
            boolean flag = true;
            long startTime = System.currentTimeMillis();
            long nowTime = startTime;
            while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("JianYang.TimeOut"))) {
                try {
                    Page pageContent = null;
                    String id = getTidByUrl(url);
                    String orgUrl = "http://www.jy0832.com/detail-%s-%s-1";
                    pageContent = getHtml(url);
                    if (null == pageContent) {

                    } else if (pageContent.getStatusCode() == 0) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                    } else if (pageContent.getStatusCode() == 200) {
                        try {
                            document = pageContent.getHtml().getDocument();
                            document.select(".aimg_tip").remove();
                            String title = document.title();
                            Element pageInfo = document.select(".ptn.hm").get(0);
                            String content = "";
                            if (null != document.select(".plhin") && document.select(".plhin").size() > 0 && !"".equals(document.select(".plhin").get(0).select(".plc>.pct").html())) {
                                int read_count = Integer.parseInt(pageInfo.select(".xi1").get(0).text());
                                int reply_count = Integer.parseInt(pageInfo.select(".xi1").get(1).text());
                                data.put("reply_count", reply_count);
                                data.put("read_count", read_count);
                                data.put("title", (title));
                                data.put("url", url);
                                page = (int) Math.ceil(((double) reply_count + 1) / 20d);//页数
                                Element e_publish_info = document.select(".favatar").get(0);
                                String publish_user = e_publish_info.select(".authi>a").text();
                                String uid = getUidByUrl(e_publish_info.select(".authi>a").attr("href"));
                                String publish_time = getTime(document.select(".plhin").get(0).select(".plc>.pi").text()) + ":00";
                                data.put("publish_time", publish_time);
                                content = document.select(".plhin").get(0).select(".plc>.pct").html();

                                JSONArray img_urls = new JSONArray();
                                Elements imgElements = document.select(".plhin").get(0).select(".plc>.pct").select("img[zoomfile]");
                                for (int imgIndex = 0; imgIndex < imgElements.size(); imgIndex++) {
                                    Element imgElement = imgElements.get(imgIndex);
                                    String img_url = imgElement.absUrl("zoomfile");
                                    img_urls.add(img_url);
                                }
                                data.put("img_urls", img_urls);
                                data.put("spider_time", DateUtils.formatDateTime(new Date()));
                                data.put("content", FileUtils.filterImg(content));
                                data.put("publish_user_id", uid);
                                data.put("publish_user", publish_user);
                                data.put("publish_user_photo", String.format("http://www.jy0832.com/uc/avatar.php?uid=%s&size=middle", new String[]{uid}));
                                data.put("id", id);

                                long tsp = DateUtils.dateToTimestamp(DateUtils.parseDate(publish_time)).getTime();
                                String filePrefix = "JianYangForum";//文件前缀
                                String fileName = filePrefix + "_" + tsp + "_" + uid + "_" + id;
                                String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-", "-");
                                String speechesId = "JianYangForum_speeches_" + publishDate + "_" + fileName;
                                File file = FileUtils.createTempFile(speechesId + "_pageNo_1", document.html().toString());
                                files.put(speechesId + "_pageNo_1", file);
                                //帖子首页的文件

                                Elements replyEntities = document.select(".plhin");
                                replyEntities.remove(0);
                                JSONArray reply_nodes = new JSONArray();

                                // 爬取第一页的回复

                                for (int i = 0; i < replyEntities.size(); i++) {
                                    try {
                                        Element e_replyUserInfo = replyEntities.get(i).select(".favatar").get(0);
                                        JSONObject reply = new JSONObject();
                                        String reply_user = e_replyUserInfo.select(".authi>a").text();
                                        String reply_user_id = getUidByUrl(e_replyUserInfo.select(".authi>a").attr("href"));
                                        String reply_time = getTime(replyEntities.get(i).select(".pti>.authi").text()) + ":00";
                                        String replyUrl = url + "#" + replyEntities.get(i).id();
                                        reply.put("publish_time", reply_time);
                                        String reply_content = replyEntities.get(i).select(".plc>.pct").html();

                                        JSONArray reply_img_urls = new JSONArray();
                                        Elements replyImgElements = replyEntities.get(i).select(".plc>.pct").select("img[zoomfile]");
                                        for (int imgIndex = 0; imgIndex < replyImgElements.size(); imgIndex++) {
                                            Element imgElement = replyImgElements.get(imgIndex);
                                            String img_url = imgElement.absUrl("zoomfile");
                                            reply_img_urls.add(img_url);
                                            System.out.println(img_url);
                                        }
                                        reply.put("content", FileUtils.filterImg(reply_content));
                                        reply.put("publish_user_id", reply_user_id);
                                        reply.put("publish_user", reply_user);
                                        reply.put("publish_user_photo", String.format("http://www.jy0832.com/uc/avatar.php?uid=%s&size=middle", new String[]{reply_user_id}));
                                        reply.put("url", replyUrl);
                                        reply.put("parent_id", id);
                                        reply.put("ancestor_id", id);
                                        reply.put("img_urls", reply_img_urls);
                                        reply_nodes.add(reply);
                                    } catch (Exception e) {
                                        logger.error("因为禁言无法获取的情况", e);
                                        continue;// 因为禁言无法获取的情况
                                    }

                                }

                                JSONArray replys = new JSONArray();
                                List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();

                                // 爬取2页之后每页的回复
                                if (page >= 2) {
                                    int threadNum = Integer.valueOf(PropertiesUtil.getVal("JianYang.ReplyThreadCount"));
                                    List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                                    // 线程池 初始化十个线程 和JDBC连接池是一个意思 实现重用
                                    ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                                    try {
                                        ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, page, threadNum);
                                        for (int i = 0; i < splitPages.size(); i++) {
                                            JianYangForumDownLoader downLoader = this.cloneDownLoader();
                                            ReplyCrawler callable = new ReplyCrawler(downLoader, splitPages.get(i), url, id, speechesId);
                                            FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                            futureTasks.add(futureTask);
                                            executorService.submit(futureTask);
                                        }
                                        int count = 0;

                                        for (FutureTask<JSONArray> futureTask : futureTasks) {
                                            JSONArray replyArrays = futureTask.get();
                                            for (int i = 0; i < replyArrays.size(); i++) {
                                                JSONObject object = replyArrays.getJSONObject(i);
                                                replys.add(object);
                                            }
                                        }
                                        if (replys.size() > 0 && replys != null) {
                                            for (int i = 0; i < replys.size(); i++) {
                                                JSONObject reply_object = replys.getJSONObject(i);
                                                reply_nodes.add(reply_object);
                                            }
                                        }
                                        if (reply_nodes.size() > 0 && reply_nodes != null) {
                                            for (int i = 0; i < reply_nodes.size(); i++) {
                                                JSONObject reply_object = reply_nodes.getJSONObject(i);
                                                Map<String, Object> map = reply_object;
                                                list.add(map);
                                            }
                                        }
                                        Collections.sort(list, new Comparator<Map<String, Object>>() {

                                            public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                                String s1 = o1.get("publish_time").toString();
                                                String s2 = o2.get("publish_time").toString();
                                                int flag = s1.compareTo(s2);
                                                return flag;
                                            }
                                        });
                                    } catch (Exception ee) {
                                        ee.printStackTrace();
                                        logger.error(ee.getMessage(), ee);
                                    } finally {
                                        executorService.shutdown();
                                    }
                                } else {
                                    if (reply_nodes.size() > 0 && reply_nodes != null) {
                                        for (int i = 0; i < reply_nodes.size(); i++) {
                                            JSONObject reply_object = reply_nodes.getJSONObject(i);
                                            Map<String, Object> map = reply_object;
                                            list.add(map);
                                        }
                                    }
                                }
                                data.put("reply_nodes", list);
                            } else {

                            }
                        } catch (Exception e) {
                            logger.error("抓取帖子出现异常", e);
                        } finally {
                            flag = false;
                        }

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                    } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                    } else {

                    }

                } catch (Exception e1) {
                    //System.out.println("异常信息：" + e1.getMessage());
                    logger.error(e1.getMessage(), e1);
                } finally {
                    if (flag == true) {
                        item = new JSONObject();
                        data = new JSONObject();
                    }
                    nowTime = System.currentTimeMillis();
                }
            }

        } catch (Exception e) {
            item = new JSONObject();
            data = new JSONObject();
            logger.error(e.getMessage(), e);
        }
        item.put("data", data);
        return item;
    }


    public JianYangForumDownLoader cloneDownLoader()
            throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        JianYangForumDownLoader loader = new JianYangForumDownLoader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    public class ReplyCrawler implements Callable<JSONArray> {
        private JianYangForumDownLoader downLoader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String speeches_id;

        @Override
        public JSONArray call() throws Exception {
            JSONArray array = new JSONArray();
            for (int k = 0; k < pages.size(); k++) {
                JSONArray arrayPage = new JSONArray();
                try {

                    boolean flag = true;
                    long startTime = System.currentTimeMillis();
                    long nowTime = startTime;
                    while (flag && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("JianYang.TimeOut"))) {
                        try {
                            // doc= get("http://www.mala.cn/forum.php",true);
                            String[] urlParam = url.split("-");
                            // System.out.println("页码："+pages.get(k));
                            String getUrl = urlParam[0] + "-" + urlParam[1] + "-" + pages.get(k) + "-" + urlParam[3];
                            Document document = null;
                            Page pageContent = null;
                            pageContent = downLoader.getHtml(getUrl);
                            if (null == pageContent) {

                            } else if (pageContent.getStatusCode() == 0) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("1")) {

                            } else if (pageContent.getStatusCode() == 200) {
                                document = pageContent.getHtml().getDocument();
                                document.select(".aimg_tip").remove();
                                File file = FileUtils.createTempFile(speeches_id + "_pageNo_" + pages.get(k), document.html().toString());
                                files.put(speeches_id + "_pageNo_" + pages.get(k), file);
                                Elements replyEntities = document.select(".plhin");
                                replyEntities.remove(0);
                                JSONArray reply_nodes = new JSONArray();
                                for (int i = 0; i < replyEntities.size(); i++) {
                                    try {
                                        Element e_replyUserInfo = replyEntities.get(i).select(".favatar").get(0);
                                        JSONObject reply = new JSONObject();
                                        String reply_user = e_replyUserInfo.select(".authi>a").text();
                                        String reply_user_id = getUidByUrl(e_replyUserInfo.select(".authi>a").attr("href"));
                                        String reply_time = getTime(replyEntities.get(i).select(".pti>.authi").text()) + ":00";
                                        String replyUrl = getUrl + "#" + replyEntities.get(i).id();
                                        reply.put("publish_time", reply_time);
                                        String reply_content = replyEntities.get(i).select(".plc>.pct").html();

                                        JSONArray reply_img_urls = new JSONArray();
                                        Elements replyImgElements = replyEntities.get(i).select(".plc>.pct").select("img[zoomfile]");
                                        for (int imgIndex = 0; imgIndex < replyImgElements.size(); imgIndex++) {
                                            Element imgElement = replyImgElements.get(imgIndex);
                                            String img_url = imgElement.absUrl("zoomfile");
                                            reply_img_urls.add(img_url);
                                            System.out.println(img_url);
                                        }

                                        reply.put("content", FileUtils.filterImg(reply_content));
                                        reply.put("publish_user_id", reply_user_id);
                                        reply.put("publish_user", reply_user);
                                        reply.put("publish_user_photo", String.format("http://www.jy0832.com/uc/avatar.php?uid=%s&size=middle", new String[]{reply_user_id}));
                                        reply.put("url", replyUrl);
                                        reply.put("parent_id", parent_id);
                                        reply.put("ancestor_id", parent_id);
                                        reply.put("img_urls", reply_img_urls);
                                        arrayPage.add(reply);

                                    } catch (Exception e) {
                                        logger.error("因为禁言无法获取的情况", e);
                                        continue;// 因为禁言无法获取的情况
                                    }
                                }
                                flag = false;
                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("3")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("4")) {

                            } else if (String.valueOf(pageContent.getStatusCode()).startsWith("5")) {

                            } else {

                            }

                        } catch (Exception e1) {
                            //System.out.println("异常信息：" + e1.getMessage());

                            logger.error(e1.getMessage(), e1);
                        } finally {
                            if (flag == false) {
                                array.addAll(arrayPage);
                            }
                            nowTime = System.currentTimeMillis();
                        }
                    }

                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                    continue;
                }
            }
            return array;
        }

        public ReplyCrawler(JianYangForumDownLoader downLoader, ArrayList<Integer> pages, String url, String parent_id, String speeches_id) {
            this.downLoader = downLoader;
            this.pages = pages;
            this.url = url;
            this.parent_id = parent_id;
            this.speeches_id = speeches_id;
        }
    }


    /**
     * @param url 帖子URL
     * @return
     * @author 夏茂轩
     * @date 2017/1/17 9:31
     * @description 根据帖子的URL提取帖子的Tid
     */
    private String getTidByUrl(String url) throws Exception {
        Pattern pattern = Pattern.compile("http://www.jy0832.com/detail-(\\d{1,10})-");
        Matcher matcher = pattern.matcher(url);
        String tid = "";
        while (matcher.find()) {
            tid = matcher.group(1);
        }
        if (tid.equals("")) {
            throw new Exception("提取失败");
        } else {
            return tid;
        }
    }

    /**
     * @param url 用户主页URL
     * @return String
     * @throws Exception 提取异常
     * @author 夏茂轩
     * @date 2017/1/17 9:31
     * @description 根据用户的主页URL来提取uid
     */
    private String getUidByUrl(String url) throws Exception {
        Pattern pattern = Pattern.compile("http://www.jy0832.com/space-uid-(\\d{1,20})");
        Matcher matcher = pattern.matcher(url);
        String uid = "";
        while (matcher.find()) {
            uid = matcher.group(1);
        }
        if (uid.equals("")) {
            throw new Exception("提取失败");
        } else {
            return uid;
        }
    }

    public static void main(String[] args) {
        JianYangForumDownLoader downLoader = new JianYangForumDownLoader();
        downLoader.crawlItem("http://www.jy0832.com/detail-271891-1-1");
    }
}
